package edu.gatech.doi.spamframework.mapping;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import org.jsoup.Jsoup;

import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.converters.ArffSaver;

import edu.gatech.doi.spamframework.core.MappingXMLFiles;
import edu.gatech.doi.spamframework.core.ModelXMLFiles;
import edu.gatech.doi.spamframework.model.ModelAttribute;
import edu.gatech.doi.spamframework.model.XMLOperator;
import edu.gatech.doi.spamframework.parsers.EmailMessage;
import edu.gatech.doi.spamframework.parsers.EmailParser;
import edu.gatech.doi.spamframework.parsers.HtmlParser;

public class WebMapper implements Mapper{

private String modelsdirectory=System.getProperties().getProperty("user.dir")+System.getProperties().getProperty("file.separator")+"models"+System.getProperties().getProperty("file.separator");
	
	private String mappingsdirectory=System.getProperties().getProperty("user.dir")+System.getProperties().getProperty("file.separator")+"mappings"+System.getProperties().getProperty("file.separator");
	private String propertiesdirectory=System.getProperties().getProperty("user.dir")+System.getProperties().getProperty("file.separator")+"properties"+System.getProperties().getProperty("file.separator");
	private String instancesdirectory=System.getProperties().getProperty("user.dir")+System.getProperties().getProperty("file.separator")+"instances"+System.getProperties().getProperty("file.separator");
	private String datasetsdirectory=System.getProperties().getProperty("user.dir")+System.getProperties().getProperty("file.separator")+"datasets"+System.getProperties().getProperty("file.separator")+"Web"+System.getProperties().getProperty("file.separator");
	
	private String personmodelfilename;
	
	private String webpagemodelfilename;
	
	private String personmappingfilename;
	private String webpagemappingfilename;
	
	private HashMap personmodelhm;
	
	private HashMap personmappinghm;
	
	private HashMap webpagemodelhm;
	
	private HashMap webpagemappinghm;
	

	
	private String webpagearfffilename;

	//0: append; 1: replace
	private int isappended=0;

	private String labelvalue;

	private int limit=0;
	
	public void init()
	{	
		  //Get the jvm heap size.  

		
		long heapSize = Runtime.getRuntime().totalMemory();  
		
		//Print the jvm heap size.  

	    System.out.println("Heap Size = " + heapSize);  

		
		// load property file for myspace
		
		String filepath0=this.propertiesdirectory+"webpage.properties";
//		System.out.println(filepath0);
		PropertiesFileOperator pfo=new PropertiesFileOperator(filepath0);
		pfo.load();
		HashMap mhm=ModelXMLFiles.getMap();
		HashMap mappinghm=MappingXMLFiles.getMap("WEBPAGE");
		this.webpagemodelfilename=(String) mhm.get("WEBPAGE");
		this.webpagemappingfilename=(String) mappinghm.get("WEBPAGE.WEBPAGE");
		this.personmodelfilename=(String) mhm.get("PERSON");
		this.webpagearfffilename=pfo.getProperty("WEBPAGEARFF");
//		this.trecpersonarfffilename=pfo.getProperty("PERSONARFF");
		this.isappended=Integer.parseInt(pfo.getProperty("ISAPPENDED"));
		this.labelvalue=pfo.getProperty("LABEL");
		this.limit=Integer.parseInt(pfo.getProperty("LIMIT"));
		if(pfo.getProperty("INSTANCESDIR")!=null)
		{
			this.instancesdirectory=pfo.getProperty("INSTANCESDIR");
		}
		
		// load models and mappings
		XMLOperator xmlo=new XMLOperator();

		
		//message models
		String filepath3=this.modelsdirectory+this.webpagemodelfilename;
		this.webpagemodelhm=new HashMap();
//		System.out.println("filepath:"+filepath);
		this.webpagemodelhm=xmlo.getModelAttrsFromXML(filepath3);
//		if(this.personmodelhm==null)
//		{
//			System.out.println("personmappinghm is null");
//		}
		String filepath4=this.mappingsdirectory+this.webpagemappingfilename;
		this.webpagemappinghm=new HashMap();
		System.out.println(filepath4);
		this.webpagemappinghm=xmlo.getMappingAttrsFromXML(filepath4);
		
		
		

		
	}
	
	

	public Instances parseWebPages(String HtmlFile)
	{
		
		// TODO Auto-generated method stub
		Instances       data = null;
		FastVector      atts;
		/**
		 * attributes list
		 */
		
		ArrayList attributelist=new ArrayList();
		/**
		 * From,AllRecipients,Subject,Date,Content-type,Message-ID,Body,
		 */
	
		Iterator attrit=this.webpagemappinghm.keySet().iterator();
		HashMap mappingattrhm=new HashMap();
		while(attrit.hasNext())
		{
			String tempattrstr=(String) attrit.next();
			attributelist.add(tempattrstr);
			MappingAttribute mappinga=(MappingAttribute) this.webpagemappinghm.get(tempattrstr);
			mappingattrhm.put(mappinga.getModelAttributeName(), mappinga);
		}
			// 1. set up attributes
			
		    atts = new FastVector();
		    /**
			 * From,AllRecipients,Subject,Date,Content-type,Message-ID,Body,
			 * 
			 */
		   Set personmodelkeyset= this.webpagemodelhm.keySet();
		   List list=new ArrayList(personmodelkeyset);
		   Collections.sort(list);
		    Iterator it=list.iterator();
		    while(it.hasNext())
		    {
		    	String tempstr=(String)it.next();
//		    	System.out.println("tempstr1="+tempstr);
		    	ModelAttribute attribute=(ModelAttribute)this.webpagemodelhm.get(tempstr.trim());
		    	String attributename=attribute.getAttributeName().trim();
//		    	System.out.println("attributename:"+attributename);
		    	int typeid=attribute.getAttributeTypeId();
		    	
		    	switch(typeid)
		    	{
		    	case 1:
		    		//-numeric
		    		
		    		atts.addElement(new Attribute(attributename));
		    		break;
		    	case 2:
		    		//-nominal
		    		FastVector attVals;
		    	    attVals = new FastVector();
		    	    ArrayList al=attribute.getNomialValues();
		    	    Iterator ital=al.iterator();
		    	    while(ital.hasNext())
		    	    {
		    	    	String alstr=(String) ital.next();
		    	    	attVals.addElement(alstr);
		    	    }
		    	    atts.addElement(new Attribute(attributename, attVals));
		    	    break;
		    		
		    	case 3:
		    		//-string
		    		//System.out.println("attributename:"+attributename);
		    		atts.addElement(new Attribute(attributename, (FastVector) null));
		    		break;
		    	case 4:
		    		//-date
		    	    atts.addElement(new Attribute(attributename, (FastVector) null));
		    	    break;
		    	}
		    }
		 // - string
		    // instance_id and label
		    Attribute instanceid=new Attribute("INSTANCE_ID", (FastVector) null);
		    Attribute label=new Attribute("LABEL", (FastVector) null);
		    atts.addElement(instanceid);
		    atts.addElement(label);
		    
		    
		 // 2. create Instances object
		    data = new Instances("WebPages", atts, 0);
		   
		    HashMap hm=this.webpageMapping(attributelist);
		    
		    //create file
		    String arfffilename=this.instancesdirectory+this.webpagearfffilename;   
		    File arfffile=new File(arfffilename);
		    if(!arfffile.exists())
		    {
		    	try {
		    		arfffile.createNewFile();
				 	this.save2arff(data, this.instancesdirectory, this.webpagearfffilename);
	 
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
		    }
		    else
		    {
		    	try {
			    	if(this.isappended==1)
		    		{
						arfffile.delete();
						arfffile.createNewFile();
					 	this.save2arff(data, this.instancesdirectory, this.webpagearfffilename);
		    		}
		    	
	    		} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
		    }
		    
		    FileWriter writer = null;
		    BufferedWriter bw=null;

			 HashMap inhm=new HashMap();
			 
 
			//HtmlParser
			 HtmlParser hp=new HtmlParser();
			 
			 
		// Now do something with the ResultSet ....
		 // convert resultset to instances
			System.out.println("converting resultset to instances");
			
			if(hp.parseHTMLDoc(HtmlFile))
			{
		
			 // 3. fill with data
			    int num=0;
			 
			
//			    	System.out.println("num0="+num);
			    	/**
					 * id name location description  url
					 */
			    	
			    	Iterator it1=attributelist.iterator();
					while(it1.hasNext())
					{
						String attstr=(String)it1.next();
						String rows = null;
						if(attstr.equalsIgnoreCase("BODY"))
						{
							rows=hp.getBodyText();
						}
						else
						{
//							System.out.println(attstr);
							rows= (String) hp.getHeaderMap().get(attstr);
						}
//						System.out.println(rows);
						ModelAttribute attribute=(ModelAttribute)hm.get(attstr);
						String attributename=attribute.getAttributeName();
//						System.out.println(attributename.trim());
//						System.out.println(rows);
						inhm.put(attributename.trim(), rows);
					}
					
				    Instance newInst = new Instance(data.numAttributes());
					for(int i=0;i<data.numAttributes();i++)
					{
					//System.out.println("i="+i);
					//	System.out.println("Attribute:"+data.attribute(i).name());
						if(inhm.containsKey(data.attribute(i).name()))
						{
							if(inhm.get(data.attribute(i).name())==null)
								continue;
						 //-numeric
						   if(data.attribute(i).isNumeric())
						   {
							   newInst.setValue(data.attribute(i),Double.valueOf((String) inhm.get(data.attribute(i).name())));
							   
						   }
						   //-nominal
						   if(data.attribute(i).isNominal())
						   {
							   MappingAttribute mapa=(MappingAttribute) mappingattrhm.get(data.attribute(i).name());
							   ModelAttribute modela=(ModelAttribute) hm.get(mapa.getAttributeName());
							   String tempstr=(String) inhm.get(data.attribute(i).name());
							   ArrayList modelal=modela.getNomialValues();
							   ArrayList mappingal=mapa.getNomialValues();
							   HashMap mappingnominal=new HashMap();
							   for(int n=0;n<modelal.size();n++)
							   {
//								   System.out.println("mapping:"+mappingal.get(n)+ modelal.get(n));
								   mappingnominal.put(mappingal.get(n), modelal.get(n));
							   }
							   
							   newInst.setValue(data.attribute(i),(String)mappingnominal.get(tempstr));
						   }
						   //-string
						   if(data.attribute(i).isString())
						   {
							   System.out.println("--"+data.attribute(i).name());
							   String tempstr=(String)inhm.get(data.attribute(i).name());
							   System.out.println(tempstr);
							   
							   newInst.setValue(data.attribute(i),tempstr);
								
		//					   vals[i] = data.attribute(i).addStringValue("This is a string!");
						   }
					   
						   //-date
						   if(data.attribute(i).isDate())
						   {
							   newInst.setValue(data.attribute(i),(String) inhm.get(data.attribute(i).name()) );
								

						   }
						}
					}
					
					
					String instanceidstr=HtmlFile;
					
					newInst.setValue(data.attribute(data.numAttributes()-2), instanceidstr);
					
					newInst.setValue(data.attribute(data.numAttributes()-1), this.labelvalue);
					
				    // add
				    data.add(newInst);
				    newInst=null;

			 
			 // 4. output data
//			    System.out.println(data);

		   
		    // first time saving 


		    	 //append new instance to the file
		    	
		
					
			    try {
			    	writer =new FileWriter(arfffilename, true);
					
				    bw=new BufferedWriter(writer);
			    	  // You are now at the end of the file,
			    	  // and can start writing new data out, e.g.
			    	 bw.newLine(); 
			    	 for(int i=0;i<data.numInstances();i++)
			    	 {
			    		 bw.write(data.instance(i).toString());
			    		 bw.newLine();
			    		 bw.flush();
			    	 }
			    	 bw.close();
			    	 writer.close();
			    	 bw=null;
			    	 writer=null;
			    	} catch (IOException ex ) {
			    	  ex.printStackTrace();
			    	}finally {                       // always close the file
			    		 if (bw != null) try {
			    		    bw.close();
			    		 } catch (IOException ioe2) {
			    		    // just ignore it
			    		 }
			    	 }
			    	// delete instances in data
			    data.delete();
			}
		
		return data;
	}
	
	@Override
	public Instances parseMessages(String msgsFile) {
		return null;
	}

	@Override
	public Instances parsePersons(String personsFile) {
		// TODO Auto-generated method stub
		return null;
	} 
	/**
	 * attributes mapping
	 * @return
	 */
	public HashMap webpageMapping(ArrayList attr)
	{

		HashMap hm=new HashMap();
		Iterator it=attr.iterator();
		while(it.hasNext())
		{
			String tempstr=(String) it.next();
//			System.out.println("personMapping tempstr="+tempstr);
			/**
			 * id, user_id, text, in_reply_to_user_id
			 */
			MappingAttribute maa=(MappingAttribute)(this.webpagemappinghm.get(tempstr));
			String modelattributename=maa.getModelAttributeName();
//			System.out.println("modelattributename:"+modelattributename);
			ModelAttribute ma=(ModelAttribute)this.webpagemodelhm.get(modelattributename);

//			System.out.println("ma.getattributename:"+ma.getAttributeName());
			hm.put(tempstr,ma);
		}
		return hm;
	}
	
	/**
	 * save instances to arff files
	 * @param in
	 * @param filedirectory
	 * @param filename
	 */
	public void save2arff(Instances in, String filedirectory, String filename)
	{
		 Instances dataSet =in;
		 ArffSaver saver = new ArffSaver();
		 saver.setInstances(dataSet);
		 try {
			saver.setFile(new File(filedirectory+System.getProperties().getProperty("file.separator")+filename));
		//	saver.setDestination(new File("./data/test.arff"));   // **not** necessary in 3.5.4 and later
			 saver.writeBatch();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		 
	}
	
	
	
	public int getIsappended() {
		return isappended;
	}


	public void setIsappended(int isappended) {
		this.isappended = isappended;
	}
	public String getLabelvalue() {
		return labelvalue;
	}
	public void setLabelvalue(String labelvalue) {
		this.labelvalue = labelvalue;
	}
	
	public void parseWebPagesFromIndexFile(String indexFileName)
	{
		File file=new File(indexFileName);
		
		if(file.exists())
		{
			try {
				FileReader fis=new FileReader(file);
				BufferedReader bis=new BufferedReader(fis);
				String line;
				int totalnum=0;
				while((line=bis.readLine())!=null)
				{
					
					this.parseWebPages(line);
					totalnum++;
					if(this.limit>0&&totalnum>this.limit)
					{
						break;
					}
				}
				
				
			} catch (FileNotFoundException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		else
		{
			System.out.println("Index File does not exist!!!");
			
		}
		
	
	}
	
	public static void main(String args[])
	{
		WebMapper wm=new WebMapper();
		 wm.init();
		 String substr2=System.getProperty("user.dir")+System.getProperties().getProperty("file.separator")+"datasets"+System.getProperties().getProperty("file.separator")+"WebbSpamCorpus"+System.getProperties().getProperty("file.separator");
		 File file=new File(substr2);
		 File[] filelist=file.listFiles();
    
    	for(int j=0;j<filelist.length;j++)
    	{
    		
    			String tempfilename=filelist[j].getPath();
    			wm.parseWebPages(tempfilename);
    		
    	}
	}
}
